from datetime import timedelta
from utils.operators.spark_sql_operator import SparkSqlOperator
from jms.dwd.oms.dwd_yl_oms_oms_waybill_incre_dt import jms_dwd__dwd_yl_oms_oms_waybill_incre_dt
from jms.dwd.tab.dwd_dependent_hi.dwd_tab_barscan_send_base_hi_03 import jms_dwd__dwd_tab_barscan_send_base_hi_03
from jms.dwd.tab.dwd_dependent_hi.dwd_tab_barscan_arrival_base_hi_03 import jms_dwd__dwd_tab_barscan_arrival_base_hi_03
from jms.dwd.tab.dwd_dependent_hi.dwd_tab_barscan_deliver_base_hi_03 import jms_dwd__dwd_tab_barscan_deliver_base_hi_03
from jms.dwd.oms.dwd_yl_oms_dispatch_waybill_base_dt import jms_dwd__dwd_yl_oms_dispatch_waybill_base_dt
from jms.dim.dim_network_whole_massage import jms_dim__dim_network_whole_massage
from jms.time_sensor.time_after_05_00 import time_after_05_00

jms_dwd__dwd_netwrok_metric_dictionary_dt = SparkSqlOperator(
    task_id='jms_dwd__dwd_netwrok_metric_dictionary_dt',
    email=['jarl.huang@jtexpress.com', 'yl_bigdata@yl-scm.com'],
    name='jms_dwd__dwd_netwrok_metric_dictionary_dt_{{ execution_date | date_add(1) | cst_ds }}',
    sql='jms/dwd/dwd_netwrok_metric_dictionary_dt/execute.sql',
    pool_slots=1,
    executor_cores=2,
    executor_memory='2G',
    driver_memory='4G',
    num_executors=20,  # spark.dynamicAllocation.enabled 为 True 时，num_executors 表示最少 Executor 数
    conf={'spark.dynamicAllocation.maxExecutors': 30,
          'spark.sql.sources.partitionOverwriteMode': 'dynamic',
          'spark.dynamicallocation.enabled': 'true',
          'spark.dynamicAllocation.cachedExecutorIdleTimeout': 120,
          'spark.shuffle.service.enabled': 'true',
          'spark.sql.shuffle.partitions': 600
          },
    hiveconf={'hive.exec.dynamic.partition': 'true',
              'hive.exec.dynamic.partition.mode': 'nonstrict',
              'hive.exec.max.dynamic.partitions.pernode': 200,
              'hive.exec.max.dynamic.partitions': 200
              },
    yarn_queue='pro',
    execution_timeout=timedelta(minutes=60),
)

jms_dwd__dwd_netwrok_metric_dictionary_dt << [
    jms_dwd__dwd_yl_oms_oms_waybill_incre_dt,
    jms_dwd__dwd_tab_barscan_send_base_hi_03,
    jms_dwd__dwd_tab_barscan_arrival_base_hi_03,
    jms_dwd__dwd_tab_barscan_deliver_base_hi_03,
    jms_dwd__dwd_yl_oms_dispatch_waybill_base_dt,
    jms_dim__dim_network_whole_massage,
    time_after_05_00
]


